import requests
from retrying import retry
from requests.exceptions import ConnectionError
'''
专门请求url地址的方法
'''

headers = {
    "User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1"
}

@retry(stop_max_attempt_number = 3) # 让被装饰的函数反复执行3次，3次全部报错才会报错，中间有一次成功，则继续执行
def _parse_url(url, type):
    try:
        response = requests.get(url = url, headers=headers, timeout=5 )
        if response.status_code == 200:
            if type=='text':
                return response.text
            elif type=='content':
                return response.content
        elif response.status_code == 302:
            #need Proxy
            pass

    except ConnectionError as e:
        print('访问出错!!!', e)

    
def parse_url(url, type):
    try:
        html_str = _parse_url(url, type)
    except:
        html_str = None
    return html_str

if __name__ == '__main__':
    print('url_parse')